{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-09T21:40:22.142492Z",
     "start_time": "2017-05-09T21:40:16.524911Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.layers import Input, Dense, Lambda, Layer\n",
    "from keras.models import Model\n",
    "from keras.layers.core import Dropout\n",
    "from keras import regularizers\n",
    "import keras\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras import backend as K\n",
    "from keras import metrics\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",35)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-05-09T21:40:24.377304Z",
     "start_time": "2017-05-09T21:40:22.150733Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "\n",
    "#y_train_labels = pd.read_pickle(\"dataset/kdd_train_2labels_y.pkl\")\n",
    "#y_train_labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "#y_test_labels = pd.read_pickle(\"dataset/kdd_test_2labels_y.pkl\")\n",
    "\n",
    "output_columns_2labels = ['is_Attack','is_Normal']\n",
    "\n",
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "x_input = kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "y_output = kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "ss = pp.StandardScaler()\n",
    "x_input = ss.fit_transform(x_input)\n",
    "\n",
    "#le = pp.LabelEncoder()\n",
    "#y_train = le.fit_transform(y_train_labels).reshape(-1, 1)\n",
    "#y_test = le.transform(y_test_labels).reshape(-1, 1)\n",
    "\n",
    "y_train = kdd_train_2labels.loc[:,output_columns_2labels].values\n",
    "\n",
    "x_train, x_valid, y_train, y_valid = ms.train_test_split(x_input, \n",
    "                              y_train, \n",
    "                              test_size=0.1)\n",
    "#x_valid, x_test, y_valid, y_test = ms.train_test_split(x_valid, y_valid, test_size = 0.4)\n",
    "\n",
    "x_test = kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "y_test = kdd_test_2labels.loc[:,output_columns_2labels].values\n",
    "\n",
    "x_test = ss.transform(x_test)\n",
    "\n",
    "#x_train = np.hstack((x_train, y_train))\n",
    "#x_valid = np.hstack((x_valid, y_valid))\n",
    "\n",
    "#x_test = np.hstack((x_test, np.random.normal(loc = 0, scale = 0.01, size = y_test.shape)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:40:17.968Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_dim = 122\n",
    "intermediate_dim = 10\n",
    "latent_dim = 32\n",
    "batch_size = 1409\n",
    "hidden_layers = 8\n",
    "classes = 2\n",
    "drop_prob = 0.2\n",
    "\n",
    "class Train:\n",
    "    def build_vae_model():\n",
    "        Train.x = Input(shape=(input_dim,))\n",
    "        \n",
    "        hidden_encoder = Train.x\n",
    "        for i in range(hidden_layers):\n",
    "            hidden_encoder = Dense(intermediate_dim, activation='relu', \n",
    "                      kernel_regularizer=keras.regularizers.l2(0.0001),\n",
    "                      activity_regularizer=keras.regularizers.l1(0.0001))(hidden_encoder)\n",
    "            \n",
    "            hidden_encoder = Dropout(rate=drop_prob)(hidden_encoder)\n",
    "\n",
    "        Train.mean_encoder = Dense(latent_dim, activation=None)(hidden_encoder)\n",
    "        Train.logvar_encoder = Dense(latent_dim, activation=None)(hidden_encoder)\n",
    "\n",
    "        def get_distrib(args):\n",
    "\n",
    "            m_e, l_e = args\n",
    "\n",
    "            # Sample epsilon\n",
    "            epsilon = np.random.normal(loc=0.0, scale=0.05, size = (batch_size, latent_dim))\n",
    "\n",
    "            # Sample latent variable\n",
    "            z = m_e + K.exp(l_e / 2) * epsilon\n",
    "            return z\n",
    "\n",
    "        z = Lambda(get_distrib,name='z_dist')([Train.mean_encoder, Train.logvar_encoder])\n",
    "\n",
    "        hidden_decoder = z\n",
    "        for i in range(hidden_layers):\n",
    "            hidden_decoder = Dense(intermediate_dim, activation=\"relu\", \n",
    "                      kernel_regularizer=keras.regularizers.l2(0.0001),\n",
    "                      activity_regularizer=keras.regularizers.l1(0.0001))(hidden_decoder)\n",
    "            hidden_decoder = Dropout(rate=drop_prob)(hidden_decoder)\n",
    "\n",
    "        Train.x_ = Dense(input_dim, activation=None, name='vae_output')(hidden_decoder)\n",
    "        \n",
    "    def build_softmax_model():\n",
    "        Train.z_ = Input(shape=(latent_dim,))\n",
    "        hidden_y = Dense(latent_dim, activation='relu', name='softmax_hidden')(Train.z_)\n",
    "        Train.y = Dense(classes, activation='softmax', name='softmax_output')(hidden_y)\n",
    "        \n",
    "def vae_loss(x, x_decoded_mean):\n",
    "    xent_loss = input_dim * keras.losses.binary_crossentropy(x, x_decoded_mean)\n",
    "    kl_loss = - 0.5 * K.sum(1 + Train.logvar_encoder - K.square(Train.mean_encoder) - K.exp(Train.logvar_encoder), axis=-1)\n",
    "    return K.abs(K.mean(xent_loss + kl_loss))\n",
    "\n",
    "\n",
    "Train.build_vae_model()\n",
    "Train.build_softmax_model()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:40:17.975Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " \n",
      " Current Layer Attributes - epochs:100 hidden layers:2 features count:4\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/100\n",
      "112720/112720 [==============================] - 9s - loss: 1.1024 - val_loss: 2.0242\n",
      "Epoch 2/100\n",
      "112720/112720 [==============================] - 2s - loss: 1.0007 - val_loss: 2.0090\n",
      "Epoch 3/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9980 - val_loss: 2.0043\n",
      "Epoch 4/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9966 - val_loss: 2.0025\n",
      "Epoch 5/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9958 - val_loss: 2.0017\n",
      "Epoch 6/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9952 - val_loss: 2.0011\n",
      "Epoch 7/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9948 - val_loss: 2.0010\n",
      "Epoch 8/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9945 - val_loss: 2.0006\n",
      "Epoch 9/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9942 - val_loss: 2.0001\n",
      "Epoch 10/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9939 - val_loss: 2.0000\n",
      "Epoch 11/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9937 - val_loss: 1.9997\n",
      "Epoch 12/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9936 - val_loss: 1.9995\n",
      "Epoch 13/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9934 - val_loss: 1.9996\n",
      "Epoch 14/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9932 - val_loss: 1.9993\n",
      "Epoch 15/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9931 - val_loss: 1.9991\n",
      "Epoch 16/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9930 - val_loss: 1.9991\n",
      "Epoch 17/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9929 - val_loss: 1.9989\n",
      "Epoch 18/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9928 - val_loss: 1.9990\n",
      "Epoch 19/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9927 - val_loss: 1.9988\n",
      "Epoch 20/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9926 - val_loss: 1.9987\n",
      "Epoch 21/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9925 - val_loss: 1.9985\n",
      "Epoch 22/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9924 - val_loss: 1.9985\n",
      "Epoch 23/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9923 - val_loss: 1.9985\n",
      "Epoch 24/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9923 - val_loss: 1.9984\n",
      "Epoch 25/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9922 - val_loss: 1.9982\n",
      "Epoch 26/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9921 - val_loss: 1.9982\n",
      "Epoch 27/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9921 - val_loss: 1.9981\n",
      "Epoch 28/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9920 - val_loss: 1.9981\n",
      "Epoch 29/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9920 - val_loss: 1.9981\n",
      "Epoch 30/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9919 - val_loss: 1.9980\n",
      "Epoch 31/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9918 - val_loss: 1.9979\n",
      "Epoch 32/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9918 - val_loss: 1.9979\n",
      "Epoch 33/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9917 - val_loss: 1.9978\n",
      "Epoch 34/100\n",
      "112720/112720 [==============================] - 3s - loss: 0.9917 - val_loss: 1.9978\n",
      "Epoch 35/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9916 - val_loss: 1.9977\n",
      "Epoch 36/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9916 - val_loss: 1.9977\n",
      "Epoch 37/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9916 - val_loss: 1.9976\n",
      "Epoch 38/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9915 - val_loss: 1.9976\n",
      "Epoch 39/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9915 - val_loss: 1.9976\n",
      "Epoch 40/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9914 - val_loss: 1.9975\n",
      "Epoch 41/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9914 - val_loss: 1.9975\n",
      "Epoch 42/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9914 - val_loss: 1.9974\n",
      "Epoch 43/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9913 - val_loss: 1.9974\n",
      "Epoch 44/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9913 - val_loss: 1.9974\n",
      "Epoch 45/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9912 - val_loss: 1.9973\n",
      "Epoch 46/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9912 - val_loss: 1.9973\n",
      "Epoch 47/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9912 - val_loss: 1.9973\n",
      "Epoch 48/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9911 - val_loss: 1.9972\n",
      "Epoch 49/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9911 - val_loss: 1.9972\n",
      "Epoch 50/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9911 - val_loss: 1.9971\n",
      "Epoch 51/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9910 - val_loss: 1.9971\n",
      "Epoch 52/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9910 - val_loss: 1.9971\n",
      "Epoch 53/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9910 - val_loss: 1.9971\n",
      "Epoch 54/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9909 - val_loss: 1.9971\n",
      "Epoch 55/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9909 - val_loss: 1.9970\n",
      "Epoch 56/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9909 - val_loss: 1.9970\n",
      "Epoch 57/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9909 - val_loss: 1.9970\n",
      "Epoch 58/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9908 - val_loss: 1.9969\n",
      "Epoch 59/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9908 - val_loss: 1.9969\n",
      "Epoch 60/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9908 - val_loss: 1.9969\n",
      "Epoch 61/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9908 - val_loss: 1.9968\n",
      "Epoch 62/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9907 - val_loss: 1.9968\n",
      "Epoch 63/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9907 - val_loss: 1.9968\n",
      "Epoch 64/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9907 - val_loss: 1.9968\n",
      "Epoch 65/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9907 - val_loss: 1.9967\n",
      "Epoch 66/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9906 - val_loss: 1.9967\n",
      "Epoch 67/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9906 - val_loss: 1.9967\n",
      "Epoch 68/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9906 - val_loss: 1.9967ss: 0\n",
      "Epoch 69/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9906 - val_loss: 1.9966\n",
      "Epoch 70/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9905 - val_loss: 1.9966\n",
      "Epoch 71/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9905 - val_loss: 1.9966\n",
      "Epoch 72/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9905 - val_loss: 1.9966\n",
      "Epoch 73/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9905 - val_loss: 1.9966\n",
      "Epoch 74/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9905 - val_loss: 1.9965\n",
      "Epoch 75/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9904 - val_loss: 1.9966\n",
      "Epoch 76/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9904 - val_loss: 1.9965\n",
      "Epoch 77/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9904 - val_loss: 1.9965\n",
      "Epoch 78/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9904 - val_loss: 1.9965\n",
      "Epoch 79/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9904 - val_loss: 1.9965\n",
      "Epoch 80/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9903 - val_loss: 1.9964\n",
      "Epoch 81/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9903 - val_loss: 1.9964\n",
      "Epoch 82/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "112720/112720 [==============================] - 2s - loss: 0.9903 - val_loss: 1.9964\n",
      "Epoch 83/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9903 - val_loss: 1.9964\n",
      "Epoch 84/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9903 - val_loss: 1.9964\n",
      "Epoch 85/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9903 - val_loss: 1.9964\n",
      "Epoch 86/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9902 - val_loss: 1.9963\n",
      "Epoch 87/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9902 - val_loss: 1.9963\n",
      "Epoch 88/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9902 - val_loss: 1.9963\n",
      "Epoch 89/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9902 - val_loss: 1.9963\n",
      "Epoch 90/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9902 - val_loss: 1.9963\n",
      "Epoch 91/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9902 - val_loss: 1.9963\n",
      "Epoch 92/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9902 - val_loss: 1.9962\n",
      "Epoch 93/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9901 - val_loss: 1.9962\n",
      "Epoch 94/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9901 - val_loss: 1.9962\n",
      "Epoch 95/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9901 - val_loss: 1.9962\n",
      "Epoch 96/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9901 - val_loss: 1.9962\n",
      "Epoch 97/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9901 - val_loss: 1.9962\n",
      "Epoch 98/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9901 - val_loss: 1.9962\n",
      "Epoch 99/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9901 - val_loss: 1.9962\n",
      "Epoch 100/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9900 - val_loss: 1.9962\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/5\n",
      "112720/112720 [==============================] - 1s - loss: 0.6918 - acc: 0.5296 - val_loss: 0.7031 - val_acc: 0.4402\n",
      "Epoch 2/5\n",
      "112720/112720 [==============================] - 0s - loss: 0.6914 - acc: 0.5319 - val_loss: 0.7088 - val_acc: 0.4308\n",
      "Epoch 3/5\n",
      "112720/112720 [==============================] - 0s - loss: 0.6913 - acc: 0.5333 - val_loss: 0.7049 - val_acc: 0.4349\n",
      "Epoch 4/5\n",
      "112720/112720 [==============================] - 0s - loss: 0.6913 - acc: 0.5333 - val_loss: 0.7042 - val_acc: 0.4350\n",
      "Epoch 5/5\n",
      "112720/112720 [==============================] - 0s - loss: 0.6913 - acc: 0.5333 - val_loss: 0.7051 - val_acc: 0.4334\n",
      "16908/22544 [=====================>........] - ETA: 0s\n",
      " Train Acc: 0.5345102921128273, Test Acc: 0.4334190897643566\n",
      " \n",
      " Current Layer Attributes - epochs:100 hidden layers:2 features count:16\n",
      "Train on 112720 samples, validate on 22544 samples\n",
      "Epoch 1/100\n",
      "112720/112720 [==============================] - 8s - loss: 0.9900 - val_loss: 1.9965\n",
      "Epoch 2/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9897 - val_loss: 1.9956\n",
      "Epoch 3/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9896 - val_loss: 1.9962\n",
      "Epoch 4/100\n",
      "112720/112720 [==============================] - 2s - loss: 0.9896 - val_loss: 1.9957\n",
      "Epoch 5/100\n",
      " 32407/112720 [=======>......................] - ETA: 2s - loss: 0.9630"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "#features_arr = [4, 16, 32, 256, 1024]\n",
    "#hidden_layers_arr = [2, 6, 10, 100]\n",
    "\n",
    "#features_arr = [4, 16, 32]\n",
    "#hidden_layers_arr = [2, 6, 10]\n",
    "\n",
    "features_arr = [4, 16, 32]\n",
    "hidden_layers_arr = [2, 4, 6]\n",
    "\n",
    "epoch_arr = [100]\n",
    "\n",
    "score = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score'])\n",
    "scores = []\n",
    "predictions = {}\n",
    "\n",
    "for e, h, f in itertools.product(epoch_arr, hidden_layers_arr, features_arr):\n",
    "    \n",
    "    print(\" \\n Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "    latent_dim = f\n",
    "    epochs = e\n",
    "    hidden_layers = h\n",
    "    \n",
    "    train_size = x_train.shape[0] - x_train.shape[0]%batch_size\n",
    "    valid_size = x_valid.shape[0] - x_valid.shape[0]%batch_size\n",
    "\n",
    "    \n",
    "    optimizer = keras.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-04, decay=0.1)\n",
    "    \n",
    "    vae_model = Model(inputs = Train.x, outputs = Train.x_)\n",
    "    vae_model.compile(optimizer = optimizer, \n",
    "                      loss = 'mse')    \n",
    "    vae_model.fit(x = x_train[:train_size,:], y = x_train[:train_size,:], \n",
    "                  shuffle=True, epochs=epochs, \n",
    "                  batch_size = batch_size, \n",
    "                  validation_data = (x_test, x_test),\n",
    "                  verbose = 1)\n",
    "    \n",
    "    z_model = Model(inputs = Train.x, outputs = vae_model.get_layer(\"z_dist\").output)\n",
    "    z_train = z_model.predict(x_train[:train_size,:], batch_size=batch_size)\n",
    "    z_valid = z_model.predict(x_valid[:valid_size,:], batch_size=batch_size)\n",
    "    z_test = z_model.predict(x_test, batch_size=batch_size)\n",
    "    \n",
    "    sm_model = Model(inputs = Train.z_, outputs = Train.y)\n",
    "    sm_model.compile(optimizer = optimizer, \n",
    "                      loss = keras.losses.categorical_crossentropy, \n",
    "                      metrics = ['accuracy'])\n",
    "    \n",
    "    sm_model.fit(x = z_train, y = y_train[:train_size,:],\n",
    "                 shuffle=True, epochs=5, \n",
    "                  batch_size = batch_size, \n",
    "                  validation_data = (z_test, y_test),\n",
    "                  verbose = 1)\n",
    "\n",
    "    \n",
    "    score_train = sm_model.evaluate(z_valid, y = y_valid[:valid_size,:],\n",
    "                               batch_size = batch_size,\n",
    "                               verbose = 1)\n",
    "    \n",
    "    score_test = sm_model.evaluate(z_test, y = y_test,\n",
    "                           batch_size = batch_size,\n",
    "                           verbose = 1)\n",
    "    \n",
    "    y_test_pred = sm_model.predict(z_test, batch_size=batch_size)\n",
    "    \n",
    "\n",
    "    y_pred = y_test_pred #np.argmax(y_test_pred[:,-2:], axis = 1)\n",
    "    \n",
    "    curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,0], \"Normal_prob\":y_pred[:,1]})\n",
    "    predictions.update({\"{}_{}_{}\".format(e,f,h):curr_pred})\n",
    "    \n",
    "    scores.append(score(e,f,h,score_train[-1], score_test[-1])) #score_test[-1]))\n",
    "    \n",
    "    print(\"\\n Train Acc: {}, Test Acc: {}\".format(score_train[-1], \n",
    "                                                  score_test[-1])  )\n",
    "    \n",
    "scores = pd.DataFrame(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:40:17.980Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "scores.sort_values(\"test_score\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:40:17.984Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(predictions).to_pickle(\"dataset/keras_vae_dense_trained_seperately_nsl_kdd_predictions.pkl\")\n",
    "scores.to_pickle(\"dataset/keras_vae_dense_trained_seperately_nsl_kdd_scores.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2017-05-09T21:40:17.990Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/33dcb1bcf3ca4a3461c4405a003a7591"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Final Hyper parameter tuning",
    "public": false
   },
   "id": "33dcb1bcf3ca4a3461c4405a003a7591"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
